From 8863d71cc8575fa0e8dd50dcb4606a9a39b89e9e Mon Sep 17 00:00:00 2001 From: "mafetter@fleming.research" Date: Fri, 25 Mar 2005 00:36:35 +0000 Subject: [PATCH] bitkeeper revision 1.1265 (42435d13hIiIzrasNZHbz13uy4ZTKg) First attempt at cleanup after merge of shadow code with unstable. Signed-off-by: michael.fetterman@cl.cam.ac.uk --- xen/arch/x86/audit.c | 33 +++++++- xen/arch/x86/domain.c | 3 + xen/arch/x86/mm.c | 143 ++++++++++++++++++----------------- xen/arch/x86/shadow.c | 12 +-- xen/include/asm-x86/domain.h | 5 ++ xen/include/asm-x86/mm.h | 14 ++-- xen/include/asm-x86/shadow.h | 9 +-- xen/include/xen/perfc_defn.h | 2 +- 8 files changed, 128 insertions(+), 93 deletions(-) diff --git a/xen/arch/x86/audit.c b/xen/arch/x86/audit.c index 97a7baf8bd..6d8b2f48a9 100644 --- a/xen/arch/x86/audit.c +++ b/xen/arch/x86/audit.c @@ -431,8 +431,7 @@ int audit_adjust_pgtables(struct domain *d, int dir, int noisy) mfn = page_to_pfn(page); page_type = page->u.inuse.type_info & PGT_type_mask; - if ( page_get_owner(page) != d ) - BUG(); + BUG_ON(page_get_owner(page) != d); page_count++; @@ -563,6 +562,31 @@ int audit_adjust_pgtables(struct domain *d, int dir, int noisy) #ifndef NDEBUG +void audit_pagelist(struct domain *d) +{ + struct list_head *list_ent; + int xenpages, totpages; + + list_ent = d->xenpage_list.next; + for ( xenpages = 0; (list_ent != &d->xenpage_list); xenpages++ ) + { + list_ent = list_ent->next; + } + list_ent = d->page_list.next; + for ( totpages = 0; (list_ent != &d->page_list); totpages++ ) + { + list_ent = list_ent->next; + } + + if ( xenpages != d->xenheap_pages || + totpages != d->tot_pages ) + { + printk("ARGH! dom %d: xen=%d %d, pages=%d %d\n", + xenpages, d->xenheap_pages, + totpages, d->tot_pages ); + } +} + void _audit_domain(struct domain *d, int flags) { void scan_for_pfn_in_mfn(struct domain *d, unsigned long xmfn, @@ -668,6 +692,8 @@ void _audit_domain(struct domain *d, int flags) spin_lock(&d->page_alloc_lock); + audit_pagelist(d); + /* PHASE 0 */ list_ent = d->page_list.next; @@ -679,8 +705,7 @@ void _audit_domain(struct domain *d, int flags) mfn = page_to_pfn(page); page_type = page->u.inuse.type_info & PGT_type_mask; - if ( page_get_owner(page) != d ) - BUG(); + BUG_ON(page_get_owner(page) != d); if ( (page->u.inuse.type_info & PGT_count_mask) > (page->count_info & PGC_count_mask) ) diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index 53a3de06b8..d06e9e7f82 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -256,6 +256,9 @@ void arch_do_createdomain(struct exec_domain *ed) ed->arch.shadow_vtable = __shadow_linear_l2_table; #ifdef __x86_64__ + ed->arch.guest_vl3table = __linear_l3_table; + ed->arch.guest_vl4table = __linear_l4_table; + d->arch.mm_perdomain_l2 = (l2_pgentry_t *)alloc_xenheap_page(); memset(d->arch.mm_perdomain_l2, 0, PAGE_SIZE); d->arch.mm_perdomain_l2[l2_table_offset(PERDOMAIN_VIRT_START)] = diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 1d79c7bb4b..04c3b61e23 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -1988,48 +1988,58 @@ int do_mmu_update( return rc; } -void update_shadow_va_mapping(unsigned long va, - unsigned long val, - struct exec_domain *ed, - struct domain *d) +/* This function assumes the caller is holding the domain's BIGLOCK + * and is running in a shadow mode + */ +int update_shadow_va_mapping(unsigned long va, + unsigned long val, + struct exec_domain *ed, + struct domain *d) { - /* This function assumes the caller is holding the domain's BIGLOCK - * and is running in a shadow mode - */ - - unsigned long sval = 0; + unsigned long l1mfn; + unsigned long spte; + int rc = 0; - l1pte_propagate_from_guest(d, &val, &sval); + check_pagetable(ed, "pre-va"); /* debug */ + shadow_lock(d); + + // This is actually overkill - we don't need to sync the L1 itself, + // just everything involved in getting to this L1 (i.e. we need + // linear_pg_table[l1_linear_offset(va)] to be in sync)... + // + __shadow_sync_va(ed, va); - if ( unlikely(__put_user(sval, ((unsigned long *)( - &shadow_linear_pg_table[l1_linear_offset(va)])))) ) +#if 1 /* keep check_pagetables() happy */ + /* + * However, the above doesn't guarantee that there's no snapshot of + * the L1 table in question; it just says that the relevant L2 and L1 + * entries for VA are in-sync. There might still be a snapshot. + * + * The checking code in _check_pagetables() assumes that no one will + * mutate the shadow of a page that has a snapshot. It's actually + * OK to not sync this page, but it seems simpler to: + * 1) keep all code paths the same, and + * 2) maintain the invariant for _check_pagetables(), rather than try + * to teach it about this boundary case. + * So we flush this L1 page, if it's out of sync. + */ + l1mfn = (l2_pgentry_val(linear_l2_table(ed)[l2_table_offset(va)]) >> + PAGE_SHIFT); + if ( mfn_out_of_sync(l1mfn) ) { - /* - * Since L2's are guaranteed RW, failure indicates either that the - * page was not shadowed, or that the L2 entry has not yet been - * updated to reflect the shadow. - */ + perfc_incrc(extra_va_update_sync); + __shadow_sync_mfn(d, l1mfn); + } +#endif /* keep check_pagetables() happy */ - /* Can't use linear_l2_table with external tables. */ - BUG_ON(shadow_mode_external(current->domain)); + if ( unlikely(__put_user(val, &l1_pgentry_val( + linear_pg_table[l1_linear_offset(va)]))) ) + return -EINVAL; - l2_pgentry_t gpde = linear_l2_table[l2_table_offset(va)]; - unsigned long gpfn = l2_pgentry_val(gpde) >> PAGE_SHIFT; + // also need to update the shadow - if (get_shadow_status(d, gpfn)) - { - unsigned long gmfn = __gpfn_to_mfn(d, gpfn); - unsigned long *gl1e = map_domain_mem(gmfn << PAGE_SHIFT); - unsigned l1_idx = l1_table_offset(va); - gl1e[l1_idx] = sval; - unmap_domain_mem(gl1e); - put_shadow_status(d); - - perfc_incrc(shadow_update_va_fail1); - } - else - perfc_incrc(shadow_update_va_fail2); - } + l1pte_propagate_from_guest(d, val, &spte); + shadow_set_l1e(va, spte, 0); /* * If we're in log-dirty mode then we need to note that we've updated @@ -2037,9 +2047,12 @@ void update_shadow_va_mapping(unsigned long va, * for this. */ if ( shadow_mode_log_dirty(d) ) - mark_dirty(d, va_to_l1mfn(va)); + mark_dirty(d, va_to_l1mfn(ed, va)); + + shadow_unlock(d); + check_pagetable(ed, "post-va"); /* debug */ - check_pagetable(d, ed->arch.guest_table, "va"); /* debug */ + return rc; } int update_grant_va_mapping(unsigned long va, @@ -2104,8 +2117,21 @@ int do_update_va_mapping(unsigned long va, * XXX When we make this support 4MB superpages we should also deal with * the case of updating L2 entries. */ - if ( unlikely(!shadow_mode_enabled(d)) ) + if ( unlikely(shadow_mode_enabled(d)) ) + { + if ( unlikely(percpu_info[cpu].foreign && + (shadow_mode_translate(d) || + shadow_mode_translate(percpu_info[cpu].foreign))) ) + { + // The foreign domain's pfn's are in a different namespace. + // There's not enough information in just a gpte to figure out + // how to (re-)shadow this entry. + // + domain_crash(); + } + rc = update_shadow_va_mapping(va, val, ed, d); + } else if ( unlikely(!mod_l1_entry(&linear_pg_table[l1_linear_offset(va)], mk_l1_pgentry(val))) ) rc = -EINVAL; @@ -2484,7 +2510,7 @@ void ptwr_flush(const int which) if ( which == PTWR_PT_ACTIVE ) { - pl2e = &linear_l2_table(ed)[ptwr_info[cpu].ptinfo[which].l2_idx]; + pl2e = &__linear_l2_table[ptwr_info[cpu].ptinfo[which].l2_idx]; *pl2e = mk_l2_pgentry(l2_pgentry_val(*pl2e) | _PAGE_PRESENT); } @@ -2502,9 +2528,9 @@ static int ptwr_emulated_update( unsigned int bytes, unsigned int do_cmpxchg) { - unsigned long sstat, pte, pfn; + unsigned long pte, pfn; struct pfn_info *page; - l1_pgentry_t ol1e, nl1e, *pl1e, *sl1e; + l1_pgentry_t ol1e, nl1e, *pl1e; struct domain *d = current->domain; /* Aligned access only, thank you. */ @@ -2581,6 +2607,8 @@ static int ptwr_emulated_update( /* Propagate update to shadow cache. */ if ( unlikely(shadow_mode_enabled(d)) ) { + BUG(); // XXX fix me... +#if 0 sstat = get_shadow_status(d, page_to_pfn(page)); if ( sstat & PSH_shadowed ) { @@ -2590,6 +2618,7 @@ static int ptwr_emulated_update( d, &l1_pgentry_val(nl1e), &l1_pgentry_val(*sl1e)); unmap_domain_mem(sl1e); } +#endif } /* Finally, drop the old PTE. */ @@ -2636,14 +2665,11 @@ int ptwr_do_page_fault(unsigned long addr) // not supported in combination with various shadow modes! ASSERT( !shadow_mode_enabled(ed->domain) ); - /* Can't use linear_l2_table with external tables. */ - BUG_ON(shadow_mode_external(ed->domain)); - /* * Attempt to read the PTE that maps the VA being accessed. By checking for * PDE validity in the L2 we avoid many expensive fixups in __get_user(). */ - if ( !(l2_pgentry_val(linear_l2_table(ed)[addr>>L2_PAGETABLE_SHIFT]) & + if ( !(l2_pgentry_val(__linear_l2_table[addr>>L2_PAGETABLE_SHIFT]) & _PAGE_PRESENT) || __get_user(pte, (unsigned long *) &linear_pg_table[l1_linear_offset(addr)]) ) @@ -2684,7 +2710,7 @@ int ptwr_do_page_fault(unsigned long addr) * Is the L1 p.t. mapped into the current address space? If so we call it * an ACTIVE p.t., otherwise it is INACTIVE. */ - pl2e = &linear_l2_table(ed)[l2_idx]; + pl2e = &__linear_l2_table[l2_idx]; l2e = l2_pgentry_val(*pl2e); which = PTWR_PT_INACTIVE; if ( (l2e >> PAGE_SHIFT) == pfn ) @@ -2824,31 +2850,6 @@ void ptwr_status(void) page = &frame_table[pfn]; } -void audit_pagelist(struct domain *d) -{ - struct list_head *list_ent; - int xenpages, totpages; - - list_ent = d->xenpage_list.next; - for ( xenpages = 0; (list_ent != &d->xenpage_list); xenpages++ ) - { - list_ent = list_ent->next; - } - list_ent = d->page_list.next; - for ( totpages = 0; (list_ent != &d->page_list); totpages++ ) - { - list_ent = list_ent->next; - } - - if ( xenpages != d->xenheap_pages || - totpages != d->tot_pages ) - { - printk("ARGH! dom %d: xen=%d %d, pages=%d %d\n", - xenpages, d->xenheap_pages, - totpages, d->tot_pages ); - } -} - #endif /* NDEBUG */ /* diff --git a/xen/arch/x86/shadow.c b/xen/arch/x86/shadow.c index 171b10ec61..fe9a009e17 100644 --- a/xen/arch/x86/shadow.c +++ b/xen/arch/x86/shadow.c @@ -1245,11 +1245,11 @@ void vmx_shadow_clear_state(struct domain *d) } unsigned long -gpfn_to_mfn_safe(struct domain *d, unsigned long gpfn) +translate_gpfn_to_mfn(struct domain *d, unsigned long gpfn) { ASSERT( shadow_mode_translate(d) ); - perfc_incrc(gpfn_to_mfn_safe); + perfc_incrc(translate_gpfn_to_mfn); unsigned long va = gpfn << PAGE_SHIFT; unsigned long phystab = pagetable_val(d->arch.phys_table); @@ -1258,7 +1258,7 @@ gpfn_to_mfn_safe(struct domain *d, unsigned long gpfn) unmap_domain_mem(l2); if ( !(l2_pgentry_val(l2e) & _PAGE_PRESENT) ) { - printk("gpfn_to_mfn_safe(d->id=%d, gpfn=%p) => 0 l2e=%p\n", + printk("translate_gpfn_to_mfn(d->id=%d, gpfn=%p) => 0 l2e=%p\n", d->id, gpfn, l2_pgentry_val(l2e)); return INVALID_MFN; } @@ -1267,12 +1267,14 @@ gpfn_to_mfn_safe(struct domain *d, unsigned long gpfn) l1_pgentry_t l1e = l1[l1_table_offset(va)]; unmap_domain_mem(l1); - printk("gpfn_to_mfn_safe(d->id=%d, gpfn=%p) => %p phystab=%p l2e=%p l1tab=%p, l1e=%p\n", +#if 0 + printk("translate_gpfn_to_mfn(d->id=%d, gpfn=%p) => %p phystab=%p l2e=%p l1tab=%p, l1e=%p\n", d->id, gpfn, l1_pgentry_val(l1e) >> PAGE_SHIFT, phystab, l2e, l1tab, l1e); +#endif if ( !(l1_pgentry_val(l1e) & _PAGE_PRESENT) ) { - printk("gpfn_to_mfn_safe(d->id=%d, gpfn=%p) => 0 l1e=%p\n", + printk("translate_gpfn_to_mfn(d->id=%d, gpfn=%p) => 0 l1e=%p\n", d->id, gpfn, l1_pgentry_val(l1e)); return INVALID_MFN; } diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h index 8e21db9916..3c37d78d6a 100644 --- a/xen/include/asm-x86/domain.h +++ b/xen/include/asm-x86/domain.h @@ -127,6 +127,11 @@ struct arch_exec_domain l2_pgentry_t *monitor_vtable; /* virtual address of monitor_table */ l1_pgentry_t *hl2_vtable; /* virtual address of hl2_table */ +#ifdef __x86_64__ + l3_pgentry_t *guest_vl3table; + l4_pgentry_t *guest_vl4table; +#endif + unsigned long monitor_shadow_ref; /* Virtual CR2 value. Can be read/written by guest. */ diff --git a/xen/include/asm-x86/mm.h b/xen/include/asm-x86/mm.h index 65d9426c26..4c03e8189e 100644 --- a/xen/include/asm-x86/mm.h +++ b/xen/include/asm-x86/mm.h @@ -4,6 +4,7 @@ #include #include +#include #include /* @@ -180,12 +181,13 @@ static inline int get_page(struct pfn_info *page, unlikely((nx & PGC_count_mask) == 0) || /* Count overflow? */ unlikely(d != _domain) ) /* Wrong owner? */ { - if ( !domain->arch.shadow_mode ) - DPRINTK("Error pfn %p: rd=%p(%d), od=%p(%d), caf=%08x, taf=%08x\n", - page_to_pfn(page), domain, (domain ? domain->id : -1), - page_get_owner(page), - (page_get_owner(page) ? page_get_owner(page)->id : -1), - x, page->u.inuse.type_info); + if ( !domain->arch.shadow_mode ) + DPRINTK("Error pfn %p: rd=%p(%d), od=%p(%d), caf=%08x, " + "taf=%08x\n", + page_to_pfn(page), domain, (domain ? domain->id : -1), + page_get_owner(page), + (page_get_owner(page) ? page_get_owner(page)->id : -1), + x, page->u.inuse.type_info); return 0; } __asm__ __volatile__( diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h index 625542b26b..739f65e48f 100644 --- a/xen/include/asm-x86/shadow.h +++ b/xen/include/asm-x86/shadow.h @@ -195,9 +195,7 @@ static inline void shadow_mode_disable(struct domain *d) ? translate_gpfn_to_mfn(_d, gpfn) \ : (gpfn) ) -#define translate_gpfn_to_mfn gpfn_to_mfn_safe - -extern unsigned long gpfn_to_mfn_safe( +extern unsigned long translate_gpfn_to_mfn( struct domain *d, unsigned long gpfn); /************************************************************************/ @@ -659,12 +657,11 @@ static inline void hl2e_propagate_from_guest( if ( unlikely((current->domain != d) && !shadow_mode_external(d)) ) { // Can't use __gpfn_to_mfn() if we don't have one of this domain's - // page tables currently installed. What a pain in the neck! - // + // page tables currently installed. // This isn't common -- it only happens during shadow mode setup // and mode changes. // - mfn = gpfn_to_mfn_safe(d, pfn); + mfn = translate_gpfn_to_mfn(d, pfn); } else mfn = __gpfn_to_mfn(d, pfn); diff --git a/xen/include/xen/perfc_defn.h b/xen/include/xen/perfc_defn.h index 2e776841e3..318882a670 100644 --- a/xen/include/xen/perfc_defn.h +++ b/xen/include/xen/perfc_defn.h @@ -98,7 +98,7 @@ PERFCOUNTER_CPU(shadow_get_page_fail, "shadow_get_page_from_l1e fails" ) PERFCOUNTER_CPU(validate_hl2e_calls, "calls to validate_hl2e_change") PERFCOUNTER_CPU(validate_hl2e_changes, "validate_hl2e makes changes") PERFCOUNTER_CPU(exception_fixed, "pre-exception fixed") -PERFCOUNTER_CPU(gpfn_to_mfn_safe, "calls to gpfn_to_mfn_safe") +PERFCOUNTER_CPU(translate_gpfn_to_mfn, "calls to translate_gpfn_to_mfn") PERFCOUNTER_CPU(remove_write_access, "calls to remove_write_access") PERFCOUNTER_CPU(remove_write_access_easy, "easy outs of remove_write_access") PERFCOUNTER_CPU(remove_write_no_work, "no work in remove_write_access") -- 2.30.2